# Imports generales
import pandas as pd
import numpy as np
import io
import math
from google.colab import files

# Visualización de datos / Gráficos
import seaborn as sns
import matplotlib.pyplot as plt
# Preprocesamiento
from sklearn.preprocessing import MinMaxScaler, StandardScaler
# Preprocesamiento / Modelado
from sklearn.decomposition import PCA
from scipy import cluster
from scipy.cluster.hierarchy import dendrogram, linkage
from sklearn.cluster import DBSCAN, AgglomerativeClustering, KMeans
from sklearn.neighbors import NearestNeighbors
from sklearn import metrics
from sklearn.metrics import silhouette_score
from sklearn.mixture import GaussianMixture

seed = 42  # Semilla aleatoria arbitraria y constante a incluir en los algoritmos estocásticos para que los experimentos sean siempre reproducibles por el profesor.
# OJO: En los experimentos estocásticos que requieran varias iteraciones con distintas semillas, podéis incorporarla como seed+1, seed+2, etc.

def upload_files (index_fields=None):
  uploaded = files.upload()
  for fn in uploaded.keys():
    print('User uploaded file "{name}" with length {length} bytes'.format(
        name=fn, length=len(uploaded[fn])))
    df = pd.read_csv(io.StringIO(uploaded[fn].decode('utf-8')), index_col = index_fields)
    return df

# Subir el conjunto de entrenamiento sin variable objetivo (dengue_features_train.csv)
train = upload_files()
print(train.shape)
train.head()

Saving dengue_features_train (1).csv to dengue_features_train (1).csv
User uploaded file "dengue_features_train (1).csv" with length 287139 bytes
(1456, 24)

train.tail()

# Obtener información sobre la BBDD dengue_features_train (tamaño, características, tipos de datos, etc.)
train.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 1456 entries, 0 to 1455
Data columns (total 24 columns):
 #   Column                                 Non-Null Count  Dtype  
---  ------                                 --------------  -----  
 0   city                                   1456 non-null   object 
 1   year                                   1456 non-null   int64  
 2   weekofyear                             1456 non-null   int64  
 3   week_start_date                        1456 non-null   object 
 4   ndvi_ne                                1262 non-null   float64
 5   ndvi_nw                                1404 non-null   float64
 6   ndvi_se                                1434 non-null   float64
 7   ndvi_sw                                1434 non-null   float64
 8   precipitation_amt_mm                   1443 non-null   float64
 9   reanalysis_air_temp_k                  1446 non-null   float64
 10  reanalysis_avg_temp_k                  1446 non-null   float64
 11  reanalysis_dew_point_temp_k            1446 non-null   float64
 12  reanalysis_max_air_temp_k              1446 non-null   float64
 13  reanalysis_min_air_temp_k              1446 non-null   float64
 14  reanalysis_precip_amt_kg_per_m2        1446 non-null   float64
 15  reanalysis_relative_humidity_percent   1446 non-null   float64
 16  reanalysis_sat_precip_amt_mm           1443 non-null   float64
 17  reanalysis_specific_humidity_g_per_kg  1446 non-null   float64
 18  reanalysis_tdtr_k                      1446 non-null   float64
 19  station_avg_temp_c                     1413 non-null   float64
 20  station_diur_temp_rng_c                1413 non-null   float64
 21  station_max_temp_c                     1436 non-null   float64
 22  station_min_temp_c                     1442 non-null   float64
 23  station_precip_mm                      1434 non-null   float64
dtypes: float64(20), int64(2), object(2)
memory usage: 273.1+ KB

# Estadística descriptiva de características numéricas climatológicas y ambientales (no nulos, media, desviación estándar, mínimo, percentiles, máximo)
train.describe(include=['float64'])

# Diagrama de barras de la variable categórica 'city'
sns.countplot(data=train, x='city')
plt.show()

# Mapear variable categórica 'city' a numérica
train['city_mapped'] = train['city'].map({'sj': 0, 'iq': 1})
print(train[['city', 'city_mapped']])

     city  city_mapped
0      sj            0
1      sj            0
2      sj            0
3      sj            0
4      sj            0
...   ...          ...
1451   iq            1
1452   iq            1
1453   iq            1
1454   iq            1
1455   iq            1

[1456 rows x 2 columns]

# También podría haber binarizado la variable categórica city con la siguiente función
#from sklearn import preprocessing
#lb = preprocessing.LabelBinarizer()
#train['city_bin'] = lb.fit_transform(train['city'])
train['city'] = train['city_mapped']
train.drop('city_mapped', axis=1, inplace=True)
train.head()

# Histogramas de las variables numéricas
train.drop(['city', 'year', 'weekofyear'], axis=1).hist(figsize=(20,15), bins=50)
plt.suptitle('Histogramas de las características', fontsize = 20)
plt.tight_layout()
plt.show()

# Diagramas de caja (boxplot) de las variables numéricas
nc = train.columns[4:23].tolist()

plt.figure(figsize=[20, 15])
plt.suptitle('Boxplots de las características', fontsize = 20)
for c in range(len(nc)):
    plt.subplot(math.ceil(len(nc)/4), 4, c+1)
    train.boxplot(nc[c])
plt.tight_layout()
plt.show()

# Visualización de los gráficos de dispersión del dataset tratado con pairplot
sns.pairplot(data=train.drop(['year', 'weekofyear'], axis=1), hue='city')
plt.suptitle('Diagramas de dispersión 2 a 2', fontsize = 20, y=1.02)
plt.show()

sns.pairplot(train[['city','reanalysis_air_temp_k','reanalysis_avg_temp_k', 'reanalysis_dew_point_temp_k','reanalysis_max_air_temp_k', 'reanalysis_min_air_temp_k', 'reanalysis_tdtr_k',
       'station_avg_temp_c', 'station_diur_temp_rng_c', 'station_max_temp_c', 'station_min_temp_c']], hue='city')
plt.suptitle('Diagramas de dispersión 2 a 2 - Temperatura', fontsize = 14, y=1.02)
plt.show()

sns.pairplot(train[['city','precipitation_amt_mm', 'reanalysis_precip_amt_kg_per_m2', 'reanalysis_relative_humidity_percent', 'reanalysis_sat_precip_amt_mm',
                    'reanalysis_specific_humidity_g_per_kg', 'station_precip_mm']], hue='city')
plt.suptitle('Diagramas de dispersión 2 a 2 - Precipitación/Humedad', fontsize = 14, y=1.02)
plt.show()

sns.pairplot(train[['city','ndvi_ne', 'ndvi_nw', 'ndvi_se', 'ndvi_sw']], hue='city')
plt.suptitle('Diagramas de dispersión 2 a 2 - Vegetación', fontsize = 14, y=1.02)
plt.show()

# Visualizo en detalle los gráficos de dispersión de que aquellas variables que tienen una relación evidente
sns.scatterplot(data=train, x='reanalysis_air_temp_k', y='reanalysis_avg_temp_k', hue='city')
plt.show()

sns.scatterplot(data=train, x='reanalysis_sat_precip_amt_mm', y='precipitation_amt_mm', hue='city')
plt.show()

sns.scatterplot(data=train, x='reanalysis_specific_humidity_g_per_kg', y='reanalysis_dew_point_temp_k', hue='city')
plt.show()

# Visualizo la matriz de correlación
numeric_data = train.select_dtypes(include=['float64'])
plt.figure(figsize=(20, 15))
sns.heatmap(numeric_data.corr(), annot=True, cmap='YlGnBu', vmin=-1, vmax=1)
plt.title('Matriz de Correlación entre Variables')
plt.show()

train.drop(['precipitation_amt_mm', 'reanalysis_dew_point_temp_k'], axis = 1, inplace = True)

train.shape

(1456, 22)

# Gráficas de evolución temporal de las características con el tiempo en años
train['week_start_date'] = pd.to_datetime(train['week_start_date'])

for col in train.columns[4:]:
    plt.figure(figsize=(10, 6))
    sns.lineplot(x='week_start_date', y=col, data=train, hue='city')
    plt.title(f'Evolución temporal de {col}')
    plt.xlabel('Año')
    plt.ylabel(col)
    plt.xticks(rotation=45)
    plt.tight_layout()
    plt.show()

# Filtro los datos para el año 2004
train_2004 = train[train['year'] == 2004]

# Grafico las series de tiempo para cada característica
for column in train_2004.columns[4:]:
    plt.figure(figsize=(10, 6))
    sns.lineplot(x='weekofyear', y=column, data=train_2004, hue='city')
    plt.title(f'Evolución temporal de {column} en 2004')
    plt.xlabel('semana')
    plt.ylabel(column)
    plt.xticks(rotation=45)
    plt.tight_layout()
    plt.show()

# Eliminar variables year y week_start_date
train.drop(['year', 'week_start_date'], axis = 1, inplace = True)
train.shape

(1456, 20)

#Divido el dataset de entrenamiento por ciudad
sj_train = train[train['city']==0]
iq_train = train[train['city']==1]

#Tamaño dataset San Juan e Iquitos
sj_train.shape, iq_train.shape

((936, 20), (520, 20))

#Estadísticas descriptivas San Juan
sj_train.describe(include=['float64'])

#Estadísticas descriptivas Iquitos
iq_train.describe(include=['float64'])

# Boxplot San Juan
nc = sj_train.columns[3:20].tolist()

plt.figure(figsize=[20, 15])
for c in range(len(nc)):
    plt.subplot(math.ceil(len(nc)/4), 4, c+1)
    sj_train.boxplot(nc[c])
plt.tight_layout()
plt.show()

# Boxplot Iquitos
nc = iq_train.columns[3:20].tolist()

plt.figure(figsize=[20, 15])
for c in range(len(nc)):
    plt.subplot(math.ceil(len(nc)/4), 4, c+1)
    iq_train.boxplot(nc[c])
plt.tight_layout()
plt.show()

# Matriz de correlación San Juan
numeric_data = sj_train.select_dtypes(include=['float64'])
plt.figure(figsize=(20, 15))
sns.heatmap(numeric_data.corr(), annot=True, cmap='YlGnBu', vmin=-1, vmax=1)
plt.title('Matriz de Correlación entre Variables de San Juan')
plt.show()

# Matriz de correlación Iquitos
numeric_data = iq_train.select_dtypes(include=['float64'])
plt.figure(figsize=(20, 15))
sns.heatmap(numeric_data.corr(), annot=True, cmap='YlGnBu', vmin=-1, vmax=1)
plt.title('Matriz de Correlación entre Variables de Iquitos')
plt.show()

train.select_dtypes(include=['float64']).isnull().sum()

# Porcentaje de valores nulos por característica
null_percentage = train.select_dtypes(include=['float64']).isnull().sum() / len(train) * 100
null_percentage.sort_values(ascending=False, inplace=True)
print(null_percentage.round(4))

ndvi_ne                                  13.3242
ndvi_nw                                   3.5714
station_diur_temp_rng_c                   2.9533
station_avg_temp_c                        2.9533
station_precip_mm                         1.5110
ndvi_sw                                   1.5110
ndvi_se                                   1.5110
station_max_temp_c                        1.3736
station_min_temp_c                        0.9615
reanalysis_sat_precip_amt_mm              0.8929
reanalysis_avg_temp_k                     0.6868
reanalysis_max_air_temp_k                 0.6868
reanalysis_min_air_temp_k                 0.6868
reanalysis_precip_amt_kg_per_m2           0.6868
reanalysis_air_temp_k                     0.6868
reanalysis_specific_humidity_g_per_kg     0.6868
reanalysis_tdtr_k                         0.6868
reanalysis_relative_humidity_percent      0.6868
dtype: float64

# Mapa de calor para visualizar valores faltantes en el dataset
sns.heatmap(train[train.columns].isnull(), cmap="viridis", cbar=False) # amarillo para valores faltantes
plt.show()

# Código para eliminar aquellas instancias que tengan más del 75% de sus características nulas

# Calculo el porcentaje de valores nulos por fila
null_percentage_per_row = train.isnull().sum(axis=1) / len(train.columns) * 100

# Identifico las instancias con más del 75% de valores nulos
instances_to_remove = train[null_percentage_per_row > 75].index

# Remuevo las instancias identificadas del dataset
train_cleaned = train.drop(instances_to_remove)

# Imprimo la forma del dataset "original" y el limpio a fines comparativos
print("Forma dataset original:", train.shape)
print("Forma dataset limpio:", train_cleaned.shape)

# Asigno el dataset limpio al original para continuar el análisis
train = train_cleaned

Forma dataset original: (1456, 20)
Forma dataset limpio: (1451, 20)

# Nuevo mapa de calor para visualizar valores faltantes resultantes en el dataset
sns.heatmap(train[train.columns].isnull(), cmap="viridis", cbar=False) # amarillo para valores faltantes
plt.show()

# Se recalcula porcentaje de valores nulos por característica
null_percentage = train.select_dtypes(include=['float64']).isnull().sum() / len(train) * 100
null_percentage.sort_values(ascending=False, inplace=True)
print(null_percentage.round(4))

ndvi_ne                                  13.0255
ndvi_nw                                   3.2391
station_diur_temp_rng_c                   2.6189
station_avg_temp_c                        2.6189
station_precip_mm                         1.1716
ndvi_sw                                   1.1716
ndvi_se                                   1.1716
station_max_temp_c                        1.0338
station_min_temp_c                        0.6203
reanalysis_sat_precip_amt_mm              0.5513
reanalysis_avg_temp_k                     0.3446
reanalysis_max_air_temp_k                 0.3446
reanalysis_min_air_temp_k                 0.3446
reanalysis_precip_amt_kg_per_m2           0.3446
reanalysis_air_temp_k                     0.3446
reanalysis_specific_humidity_g_per_kg     0.3446
reanalysis_tdtr_k                         0.3446
reanalysis_relative_humidity_percent      0.3446
dtype: float64

# IMPUTAR VALORES FALTANTES USANDO INFORMACIÓN DEL DOMINIO DEL PROBLEMA

# La temperatura promedio de temperaturas en grado centígrados en la estación meteorológica las puedo obtener como el promedio de las temperaturas máximas y mínimas
train['station_avg_temp_c'] = train['station_avg_temp_c'].fillna((train['station_min_temp_c'] + train['station_max_temp_c']) / 2)

# Continuo imputando valores faltantes de temperaturas en grados centígrados usando las mismas variables del grupo de reanalysis, convirtiendo los grados Kelvin a centígrados ( 0°C = 273.15 K)
train['station_avg_temp_c'] = train['station_avg_temp_c'].fillna((train['reanalysis_avg_temp_k'] - 273.15))
train['station_max_temp_c'] = train['station_max_temp_c'].fillna((train['reanalysis_max_air_temp_k'] - 273.15))
train['station_min_temp_c'] = train['station_min_temp_c'].fillna((train['reanalysis_min_air_temp_k'] - 273.15))
train['station_diur_temp_rng_c'] = train['station_diur_temp_rng_c'].fillna((train['reanalysis_tdtr_k'] - 273.15))

# Para las variables de precipitación con las mismas unidades de medición, se puede usar la mediana por ciudad y semana del año, considerando la estacionalidad de las lluvias.
# Se usa la mediana y no la media, porque como se vio en EDA la distribución de estas variables es altamente sesgada.
for col in ['reanalysis_sat_precip_amt_mm', 'station_precip_mm']:
    train[col] = train.groupby(['city', 'weekofyear'])[col].transform(lambda x: x.fillna(x.median()))

train.isnull().sum()

# PARA EL RESTO DE LAS VARIABLES SE PUEDE USAR LA INTERPOLACIÓN LINEAL, COMO SE PROPUSO INICIALMENTE
for col in train.columns:
  if train[col].isnull().any():
    train[col] = train[col].interpolate(method='linear')

# Verificar si quedan valores nulos
train.isnull().sum()

# Función para remover los outliers de mi dataframe
def remove_outliers(df):
    df_no_outliers = df.drop(['city', 'weekofyear'], axis=1)
    for col in df_no_outliers.columns:
        Q1 = df_no_outliers[col].quantile(0.25)
        Q3 = df_no_outliers[col].quantile(0.75)
        IQR = Q3 - Q1
        lower_bound = Q1 - 1.5 * IQR
        upper_bound = Q3 + 1.5 * IQR
        df_no_outliers = df_no_outliers[(df_no_outliers[col] >= lower_bound) & (df_no_outliers[col] <= upper_bound)]
    return df_no_outliers

remove_outliers(train)

train_prepro = train.copy()

train_prepro.describe(include=['float64'])

# NDVI
train_prepro['avg_ndvi'] = train_prepro[['ndvi_ne','ndvi_nw','ndvi_se','ndvi_sw']].mean(axis=1)
train_prepro['rng_ndvi'] = (train_prepro[['ndvi_ne','ndvi_nw','ndvi_se','ndvi_sw']].max(axis=1) - train_prepro[['ndvi_ne','ndvi_nw','ndvi_se','ndvi_sw']].min(axis=1))

# Temperaturas
train_prepro['avg_air_temp'] = ((train_prepro['reanalysis_avg_temp_k']- 273.15) + (train_prepro['reanalysis_air_temp_k']- 273.15) +  train_prepro['station_avg_temp_c']) / 3
train_prepro['avg_temp_diur_rng'] = (train_prepro['reanalysis_tdtr_k'] + train_prepro['station_diur_temp_rng_c']) / 2

# Precipitación
train_prepro['precip_avg'] = train_prepro[['reanalysis_precip_amt_kg_per_m2','station_precip_mm']].mean(axis=1)

train_prepro.columns

Index(['city', 'weekofyear', 'ndvi_ne', 'ndvi_nw', 'ndvi_se', 'ndvi_sw',
       'reanalysis_air_temp_k', 'reanalysis_avg_temp_k',
       'reanalysis_max_air_temp_k', 'reanalysis_min_air_temp_k',
       'reanalysis_precip_amt_kg_per_m2',
       'reanalysis_relative_humidity_percent', 'reanalysis_sat_precip_amt_mm',
       'reanalysis_specific_humidity_g_per_kg', 'reanalysis_tdtr_k',
       'station_avg_temp_c', 'station_diur_temp_rng_c', 'station_max_temp_c',
       'station_min_temp_c', 'station_precip_mm', 'avg_ndvi', 'rng_ndvi',
       'avg_air_temp', 'avg_temp_diur_rng', 'precip_avg'],
      dtype='object')

train_prepro.head()

# Visualizo nuevamente la matriz de correlación
numeric_data = train_prepro.select_dtypes(include=['float64'])
plt.figure(figsize=(20, 15))
sns.heatmap(numeric_data.corr(), annot=True, cmap='YlGnBu', vmin=-1, vmax=1)
plt.title('Matriz de Correlación entre Variables')
plt.show()

train_prepro.drop(['avg_air_temp', 'avg_temp_diur_rng'], axis = 1, inplace = True)

# Transpongo df preprocesado
df_feat = train_prepro.transpose()
df_feat

# 1ro) Estandarizo porque el filtrado puede haber cambiado la distribución
train_prepro_st = StandardScaler().fit_transform(train_prepro)

# 2do) Aplico PCA
pca = PCA()

#San Juan
pca.fit(train_prepro_st)
X_train_prepro_pca = pca.transform(train_prepro_st)

#Plot PCA
plt.figure(figsize=(10, 8))
plt.plot(np.cumsum(pca.explained_variance_ratio_))
plt.xlabel('Nro de componentes PCA')
plt.ylabel('Varianza explicada acumulada')
plt.title('Dataset PCA')
plt.grid(True)
plt.show()

# Normalización
scaler = StandardScaler()
features_norm = scaler.fit_transform(df_feat)

# Análisis de componentes principales (PCA)
estimator = PCA (n_components=2)
X_pca = estimator.fit_transform(features_norm)
names = df_feat.index
print("Componentes lineales:\n", estimator.components_)
print("\nRatio de variabilidad: ", estimator.explained_variance_ratio_, "\n")

# Visualización
plt.figure(figsize=(8, 6))
plt.scatter(X_pca[:, 0], X_pca[:, 1])
plt.xlabel('Componente Principal 1')
plt.ylabel('Componente Principal 2')
plt.title('PCA de 2 Componentes Dataset Preprocesado/Traspuesto/Estandarizado')
for i, name in enumerate(names):
    plt.annotate(name, (X_pca[i, 0], X_pca[i, 1]))
plt.show()

Componentes lineales:
 [[ 0.02653517  0.02651319  0.02659353 ...  0.02653045  0.02653736
   0.02656378]
 [-0.01428828 -0.01883151 -0.00447634 ... -0.00429071  0.00149504
  -0.00398964]]

Ratio de variabilidad:  [0.97326468 0.01377442]

# Calculo de la matriz de similitud
#http://docs.scipy.org/doc/scipy/reference/cluster.html
from scipy import cluster
import sklearn.metrics
dist = sklearn.metrics.DistanceMetric.get_metric('euclidean')
matdist= dist.pairwise(features_norm)

# Visualización de la matriz de distancias entre características
import seaborn as sns; sns.set()
ax = sns.heatmap(matdist,vmin=0, vmax=1, yticklabels = names, xticklabels = names)
plt.title("Matriz de Distancias entre características")

Text(0.5, 1.0, 'Matriz de Distancias entre características')

# Construcción del dendograma
# http://docs.scipy.org/doc/scipy/reference/generated/scipy.cluster.hierarchy.linkage.html#scipy.cluster.hierarchy.linkage
clusters = cluster.hierarchy.linkage(matdist, method = 'average')
# http://docs.scipy.org/doc/scipy-0.14.0/reference/generated/scipy.cluster.hierarchy.dendrogram.html
cluster.hierarchy.dendrogram(clusters, color_threshold = 210, labels = names , leaf_rotation=90) #210 es el 70% de la distancia máxima
plt.show()

<ipython-input-55-b98f95367927>:3: ClusterWarning: The symmetric non-negative hollow observation matrix looks suspiciously like an uncondensed distance matrix
  clusters = cluster.hierarchy.linkage(matdist, method = 'average')

cluster.hierarchy.dendrogram(clusters, color_threshold = 35, labels = names , leaf_rotation=90) #210 es el 70% de la distancia máxima
plt.show()

# Se realiza el corte para el umbral definido
cut = 35
labels = cluster.hierarchy.fcluster(clusters, cut , criterion = 'distance')

#plotting orginal points with color related to label
plt.scatter(X_pca[:,0], X_pca[:,1], c=labels,s=50, cmap="rainbow")
for i in range(len(X_pca)):
    plt.text(X_pca[i][0], X_pca[i][1], names[i])

plt.grid()
plt.show()

# Definimos el valor de minPts
minPts = 2

# Calculamos la matriz de distancias utilizando NearestNeighbors para encontrar los vecinos más cercanos para cada punto.
neighbors = NearestNeighbors(n_neighbors=minPts)
neighbors_fit = neighbors.fit(features_norm)
distances, indices = neighbors_fit.kneighbors(features_norm)
names=df_feat.index

# Ordenamos las distancias y las graficamos para determinar el valor de eps.
distances = np.sort(distances, axis=0)
distances = distances[:, 1]  # Tomamos la distancia al minPts-ésimo vecino más cercano
plt.plot(distances)
plt.xlabel("Puntos ordenados por distancia")
plt.ylabel("Distancia al minPts-ésimo vecino")
plt.title("Método del codo para DBSCAN")
plt.show()

# Observamos el gráfico anterior y seleccionamos un rango de valores para eps
for eps in np.arange(10, 20, 0.5):
  db = DBSCAN(eps, min_samples=minPts).fit(features_norm)
  core_samples_mask = np.zeros_like(db.labels_, dtype=bool)
  core_samples_mask[db.core_sample_indices_] = True
  labels = db.labels_
  n_clusters_ = len(set(labels)) - (1 if -1 in labels else 0)
  n_outliers = list(labels).count(-1)
  print ("%6.2f, %d, %d" % (eps, n_clusters_, n_outliers))

# Seleccionamos eps y aplicamos DBSCAN
db = DBSCAN(eps=13, min_samples=minPts).fit(features_norm)
labels = db.labels_
labels

# Usamos PCA para visualizar los resultados
plt.scatter(X_pca[:,0], X_pca[:,1], c=labels,s=50, cmap="rainbow")
for i in range(len(X_pca)):
  plt.text(X_pca[i][0], X_pca[i][1], names[i][0:6])
plt.grid()
plt.show()

# Identificamos los elementos que son outliers
df_feat['dbscan_group'] = labels
df_feat[df_feat['dbscan_group'] == -1]

 10.00, 3, 3
 10.50, 3, 3
 11.00, 3, 3
 11.50, 3, 3
 12.00, 3, 3
 12.50, 3, 2
 13.00, 2, 1
 13.50, 2, 1
 14.00, 2, 1
 14.50, 2, 1
 15.00, 2, 1
 15.50, 2, 1
 16.00, 2, 1
 16.50, 2, 1
 17.00, 2, 1
 17.50, 2, 0
 18.00, 2, 0
 18.50, 2, 0
 19.00, 2, 0
 19.50, 2, 0

# Estandarizamos los datos
train_std = StandardScaler().fit_transform(train_prepro.drop(['city', 'weekofyear'], axis=1))

distortions = []
silhouettes = []

for i in range(2, 20):
    km = KMeans(i, init='k-means++', n_init=10, max_iter=300, tol=1e-4, random_state=seed)
    clustering = km.fit_predict(train_std)
    distortions.append(km.inertia_)
    silhouettes.append(metrics.silhouette_score(train_std, clustering))

plt.plot(range(2,20), distortions, marker='o')
plt.xticks(range(2, 20))
plt.xlabel('K')
plt.ylabel('Distortion')
plt.grid(True)
plt.show()

# "Satopää et al.: Finding a “Kneedle” in a Haystack: Detecting Knee Points in System Behavior"
!pip install kneed
import kneed
from kneed import KneeLocator

Requirement already satisfied: kneed in /usr/local/lib/python3.11/dist-packages (0.8.5)
Requirement already satisfied: numpy>=1.14.2 in /usr/local/lib/python3.11/dist-packages (from kneed) (1.26.4)
Requirement already satisfied: scipy>=1.0.0 in /usr/local/lib/python3.11/dist-packages (from kneed) (1.13.1)

# Encontrar el punto de inflexión
kneedle = KneeLocator(range(2, 20), distortions[:20], curve='convex', direction='decreasing')
elbow_point = kneedle.elbow
print('Elbow: ', elbow_point)
kneedle.plot_knee()

Elbow:  6

plt.plot(range(2,20), silhouettes , marker='o', linestyle='--', color='red')
plt.xticks(range(2, 20))
plt.title("Métrica de Silueta para KMeans")
plt.xlabel('K')
plt.ylabel('Silhouette')
plt.grid(True)
plt.show()

# Métricas de silueta por K
print("Silhouette:")
print([round(silhouette, 2) for silhouette in silhouettes])

Silhouette:
[0.32, 0.27, 0.26, 0.27, 0.25, 0.23, 0.22, 0.17, 0.17, 0.16, 0.16, 0.16, 0.16, 0.15, 0.15, 0.15, 0.15, 0.15]

# Visualización de K-Means con PCA para n clusteres
def viz_kmeans_pca(data, n_clusters_list):
    pca = PCA(n_components=2)
    data_pca = pca.fit_transform(data)

    for n_clusters in n_clusters_list:
        kmeans = KMeans(n_clusters=n_clusters, init='k-means++', n_init=10, max_iter=300, random_state=seed)
        cluster_labels = kmeans.fit_predict(data)

        plt.figure(figsize=(8, 6))
        plt.scatter(data_pca[:, 0], data_pca[:, 1], c=cluster_labels, cmap='viridis', s=30)

        # Plot centroids
        centroids_pca = pca.transform(kmeans.cluster_centers_)
        plt.scatter(centroids_pca[:, 0], centroids_pca[:, 1], marker='X', s=100, color='r', label='Centroids')

        plt.xlabel('PC1')
        plt.ylabel('PC2')
        plt.title(f'Clustering K-Means con PCA (n_clusters = {n_clusters})')
        plt.legend()
        plt.show()

viz_kmeans_pca(train_std, [2, 6])

# Referencias: https://github.com/lakithasahan/Data-Anormaly-Detection, https://github.com/fariha23/recommendation_system_creditcard

# Clustering jerárquico
plt.figure(figsize=(10, 8))
plt.title("Dendograma")
plt.xlabel('índice de la instancia')
plt.ylabel('distancia euclídea')
Z = linkage(train_std, method='ward', metric='euclidean')
dendrogram(
    Z,
    leaf_rotation=90.,
    leaf_font_size=8.,
)
plt.show()

from scipy.cluster.hierarchy import cophenet
from scipy.spatial.distance import pdist

# pdist calcula las distancias entre elementos de nuestro dataset usando la
# métrica indicada (euclideana por defecto)
c, coph_dists = cophenet(Z, pdist(train_std))
c

0.6182010449940186

# Nos quedamos con las últimas p
plt.title('Hierarchical Clustering Dendrograma (truncado)')
plt.xlabel('índice de la instancia')
plt.ylabel('distancia euclídea')
dendrogram(
    Z,
    truncate_mode='lastp',  # mostrar solo las últimas p uniones
    p=12,  # definimos p
    leaf_rotation=90.,
    leaf_font_size=12.,
    show_contracted=True,
)
plt.show()

import seaborn as sns
sns.set(color_codes=True)
sns.clustermap(train_std, standard_scale=1)

/usr/local/lib/python3.11/dist-packages/seaborn/matrix.py:560: UserWarning: Clustering large matrix with scipy. Installing `fastcluster` may give better performance.
  warnings.warn(msg)
/usr/local/lib/python3.11/dist-packages/seaborn/matrix.py:560: UserWarning: Clustering large matrix with scipy. Installing `fastcluster` may give better performance.
  warnings.warn(msg)

<seaborn.matrix.ClusterGrid at 0x7c9e3e01ad90>

cluster = AgglomerativeClustering(n_clusters=3, metric='euclidean', linkage='ward') # entre 2 y 4
clusters = cluster.fit_predict(train_std)
print(clusters)
#plt.figure(figsize=(10, 7))
#plt.scatter(train_std[:,0], train_std[:,1], c=cluster.labels_, cmap='rainbow')

[2 2 2 ... 0 0 0]

plt.figure(figsize=(10, 7))
plt.scatter(train_std[:,0], train_std[:,1], c=cluster.labels_, cmap='rainbow')

<matplotlib.collections.PathCollection at 0x7c9e37b4db10>

# Aplico clustering jerárquico sin definir clusters
Z = linkage(train_std, method='ward', metric='euclidean')

# Pruebo diferentes cortes en el dendrograma
range_n_clusters = list(range(2, 10))
silueta_scores = []

for n_clusters in range_n_clusters:
    clustering = AgglomerativeClustering(n_clusters=n_clusters, linkage='ward')
    labels = clustering.fit_predict(train_std)
    score = silhouette_score(train_std, labels)
    silueta_scores.append(score)

# Grafico la silueta para elegir el número de clusters óptimo
plt.figure(figsize=(8, 5))
plt.plot(range_n_clusters, silueta_scores, marker='o', linestyle='--', color='r')
plt.xlabel('Número de clusters')
plt.ylabel('Silhouette')
plt.title('Métrica de Silueta para Clustering Jerárquico Aglomerativo')
plt.show()

# Mejor número de clusters basado en la silueta
best_n_clusters = range_n_clusters[np.argmax(silueta_scores)]
print(f"El número de clusters óptimo según la silueta es: {best_n_clusters}")

# Métricas de silueta por nro. de clusteres
print("Silhouette:")
print([round(silhouette, 2) for silhouette in silhouettes])

El número de clusters óptimo según la silueta es: 2
Silhouette:
[0.32, 0.27, 0.26, 0.27, 0.25, 0.23, 0.22, 0.17, 0.17, 0.16, 0.16, 0.16, 0.16, 0.15, 0.15, 0.15, 0.15, 0.15]

def find_optimal_eps(X):
    neighbors = NearestNeighbors(n_neighbors=2)
    neighbors_fit = neighbors.fit(X)
    distances, _ = neighbors_fit.kneighbors(X)
    distances = np.sort(distances[:, 1])

    import matplotlib.pyplot as plt
    plt.plot(distances)
    plt.xlabel("Puntos ordenados")
    plt.ylabel("Distancia al vecino más cercano")
    plt.title("Método del codo para DBSCAN")
    plt.show()

find_optimal_eps(train_std)

silhouettes = []

for i in range(2, 30):
    dbscan = DBSCAN(eps=3, min_samples= i)
    clustering = dbscan.fit_predict(train_std)
    silhouettes.append(metrics.silhouette_score(train_std, clustering))

plt.plot(range(2,30), silhouettes , marker='o', linestyle='--', color='red')
plt.xticks(range(2, 30))
plt.title("Métrica de Silueta para DBSCAN")
plt.xlabel('minpts')
plt.ylabel('Silhouette')
plt.grid(True)
plt.show()

# Métricas de silueta por K
print("Silhouette:")
print([round(silhouette, 4) for silhouette in silhouettes])

Silhouette:
[0.177, 0.319, 0.3258, 0.3245, 0.3502, 0.3497, 0.3059, 0.3056, 0.3056, 0.3038, 0.3033, 0.3001, 0.299, 0.299, 0.2986, 0.2986, 0.2986, 0.298, 0.298, 0.2975, 0.2973, 0.2973, 0.2957, 0.2957, 0.2933, 0.2925, 0.2895, 0.2896]

dbscan = DBSCAN(eps=3, min_samples = 6)
clusters = dbscan.fit_predict(train_std)
np.unique(clusters)

array([-1,  0])

# PCA para visualización
pca = PCA(n_components=2)
train_pca = pca.fit_transform(train_std)

# Graficar clusters
plt.figure(figsize=(10, 7))
plt.scatter(train_pca[:, 0], train_pca[:, 1], c=clusters, cmap='viridis', s=5)
plt.xlabel("PC1")
plt.ylabel("PC2")
plt.title("Clusters de DBSCAN con PCA (2D)")
plt.colorbar(label="Cluster")
plt.show()

def find_best_dbscan_params(X, eps_range, min_samples_range):
    """
    Encuentra los mejores parámetros (eps y min_samples) para DBSCAN optimizando el coeficiente de silueta.

    Args:
        X: Los datos para el clustering.
        eps_range: Una lista o tupla con los valores de eps a probar.
        min_samples_range: Una lista o tupla con los valores de min_samples a probar.

    Returns:
        Una tupla con:
            - El mejor valor de eps.
            - El mejor valor de min_samples.
            - El mejor coeficiente de silueta.
    """
    best_eps = None
    best_min_samples = None
    best_silhouette = -1  # Inicializamos con un valor bajo

    for eps in eps_range:
        for min_samples in min_samples_range:
            dbscan = DBSCAN(eps=eps, min_samples=min_samples)
            labels = dbscan.fit_predict(X)

            # Ignorar los casos donde solo hay un cluster o todos los puntos son ruido
            if len(set(labels)) > 1 and np.any(labels != -1):
                silhouette = silhouette_score(X, labels)
                if silhouette > best_silhouette:
                    best_silhouette = silhouette
                    best_eps = eps
                    best_min_samples = min_samples

    return best_eps, best_min_samples, best_silhouette

find_best_dbscan_params(train_std, np.arange(0.5, 20, 0.5), range(2,40))

# Referencia: https://github.com/tenyearstranger/DM_h y Gemini

(7.0, 2, 0.5950047264446199)

dbscan = DBSCAN(eps=7, min_samples = 2)
clusters = dbscan.fit_predict(train_std)
np.unique(clusters)

array([-1,  0])

k_range = range(2, 20)
silhouettes = []
aics = []

for k in k_range:
    gmm = GaussianMixture(n_components=k, random_state=seed)
    gmm_labels = gmm.fit_predict(train_std)
    silhouette_gmm = metrics.silhouette_score(train_std, gmm_labels)
    silhouettes.append(silhouette_gmm)
    aics.append(gmm.aic(train_std))

# Gráfico métrica de silueta
plt.figure()
plt.plot(k_range, silhouettes, marker='o', linestyle='--', color='red')
plt.title("Métrica de Silueta para GMM")
plt.xlabel("Nº de componentes (k)")
plt.ylabel("Silhouette")
plt.grid(True)
plt.show()

print("Silhouette:", [round(s,3) for s in silhouettes])

best_k_sil = k_range[np.argmax(silhouettes)]
print("N° de componentes óptimo =", best_k_sil)

Silhouette: [0.319, 0.249, 0.267, 0.245, 0.216, 0.156, 0.113, 0.131, 0.121, 0.051, 0.055, 0.072, 0.088, 0.087, 0.073, 0.071, 0.076, 0.071]
N° de componentes óptimo = 2

# Visualizamos con el mejor k según Silueta
gmm_final = GaussianMixture(n_components=best_k_sil, random_state=seed)
labels_gmm = gmm_final.fit_predict(train_std)

# PCA para visualización
pca = PCA(n_components=2)
train_pca = pca.fit_transform(train_std)

# Graficar clusters
plt.figure(figsize=(10, 7))
plt.scatter(train_pca[:, 0], train_pca[:, 1], c=labels_gmm, cmap='viridis', s=10)
plt.xlabel("PC1")
plt.ylabel("PC2")
plt.title(f"Clustering GMM (k={best_k_sil}) con PCA (2D)")
plt.colorbar(label="k")
plt.show()

probabilities = gmm_final.predict_proba(train_std)
probabilities[:20]

array([[1.00000000e+00, 1.68041015e-33],
       [1.00000000e+00, 9.40548469e-27],
       [1.00000000e+00, 6.13155564e-25],
       [1.00000000e+00, 2.11710360e-25],
       [1.00000000e+00, 1.90937644e-20],
       [1.00000000e+00, 6.75221689e-23],
       [1.00000000e+00, 1.85274592e-24],
       [1.00000000e+00, 1.67994056e-18],
       [1.00000000e+00, 6.80114646e-21],
       [1.00000000e+00, 3.87720647e-24],
       [1.00000000e+00, 5.41380610e-22],
       [1.00000000e+00, 3.59747012e-20],
       [1.00000000e+00, 1.57296959e-23],
       [1.00000000e+00, 3.66489151e-27],
       [1.00000000e+00, 2.16102575e-26],
       [1.00000000e+00, 7.88918355e-18],
       [1.00000000e+00, 8.67361800e-24],
       [1.00000000e+00, 5.85920833e-24],
       [1.00000000e+00, 2.90631929e-21],
       [1.00000000e+00, 8.26051175e-35]])

np.unique(probabilities)

array([0.00000000e+000, 2.00247896e-302, 7.98576139e-301, ...,
       1.00000000e+000, 1.00000000e+000, 1.00000000e+000])

# Gráfico métrica de AIC
plt.figure()
plt.plot(k_range, aics, marker='o', color='b')
plt.title("Métrica AIC para GMM")
plt.xlabel("Nº de componentes (k)")
plt.ylabel("AIC")
plt.grid(True)
plt.show()

best_k_aic = k_range[np.argmin(aics)]
print("N° AIC óptimo =", best_k_aic)

N° AIC óptimo = 11

# Visualizamos con el mejor k según AIC
gmm_final = GaussianMixture(n_components=best_k_aic, random_state=seed)
labels_gmm = gmm_final.fit_predict(train_std)

# PCA para visualización
pca = PCA(n_components=2)
train_pca = pca.fit_transform(train_std)

# Graficar clusters
plt.figure(figsize=(10, 7))
plt.scatter(train_pca[:, 0], train_pca[:, 1], c=labels_gmm, cmap='Set3', s=10)
plt.xlabel("PC1")
plt.ylabel("PC2")
plt.title(f"Clustering GMM (k={best_k_aic}) con PCA (2D)")
plt.colorbar(label="k")
plt.show()

gmm_final = GaussianMixture(n_components=4, random_state=seed)
labels_gmm = gmm_final.fit_predict(train_std)

# PCA para visualización
pca = PCA(n_components=2)
train_pca = pca.fit_transform(train_std)

# Graficar clusters
plt.figure(figsize=(10, 7))
plt.scatter(train_pca[:, 0], train_pca[:, 1], c=labels_gmm, cmap='viridis', s=10)
plt.xlabel("PC1")
plt.ylabel("PC2")
plt.title(f"Clustering GMM (k=4) con PCA (2D)")
plt.colorbar(label="k")
plt.show()

	city	year	weekofyear	week_start_date	ndvi_ne	ndvi_nw	ndvi_se	ndvi_sw	precipitation_amt_mm	reanalysis_air_temp_k	...	reanalysis_precip_amt_kg_per_m2	reanalysis_relative_humidity_percent	reanalysis_sat_precip_amt_mm	reanalysis_specific_humidity_g_per_kg	reanalysis_tdtr_k	station_avg_temp_c	station_diur_temp_rng_c	station_max_temp_c	station_min_temp_c	station_precip_mm
0	sj	1990	18	1990-04-30	0.122600	0.103725	0.198483	0.177617	12.42	297.572857	...	32.00	73.365714	12.42	14.012857	2.628571	25.442857	6.900000	29.4	20.0	16.0
1	sj	1990	19	1990-05-07	0.169900	0.142175	0.162357	0.155486	22.82	298.211429	...	17.94	77.368571	22.82	15.372857	2.371429	26.714286	6.371429	31.7	22.2	8.6
2	sj	1990	20	1990-05-14	0.032250	0.172967	0.157200	0.170843	34.54	298.781429	...	26.10	82.052857	34.54	16.848571	2.300000	26.714286	6.485714	32.2	22.8	41.4
3	sj	1990	21	1990-05-21	0.128633	0.245067	0.227557	0.235886	15.36	298.987143	...	13.90	80.337143	15.36	16.672857	2.428571	27.471429	6.771429	33.3	23.3	4.0
4	sj	1990	22	1990-05-28	0.196200	0.262200	0.251200	0.247340	7.52	299.518571	...	12.20	80.460000	7.52	17.210000	3.014286	28.942857	9.371429	35.0	23.9	5.8

	city	year	weekofyear	week_start_date	ndvi_ne	ndvi_nw	ndvi_se	ndvi_sw	precipitation_amt_mm	reanalysis_air_temp_k	...	reanalysis_precip_amt_kg_per_m2	reanalysis_relative_humidity_percent	reanalysis_sat_precip_amt_mm	reanalysis_specific_humidity_g_per_kg	reanalysis_tdtr_k	station_avg_temp_c	station_diur_temp_rng_c	station_max_temp_c	station_min_temp_c	station_precip_mm
1451	iq	2010	21	2010-05-28	0.342750	0.318900	0.256343	0.292514	55.30	299.334286	...	45.00	88.765714	55.30	18.485714	9.800000	28.633333	11.933333	35.4	22.4	27.0
1452	iq	2010	22	2010-06-04	0.160157	0.160371	0.136043	0.225657	86.47	298.330000	...	207.10	91.600000	86.47	18.070000	7.471429	27.433333	10.500000	34.7	21.7	36.6
1453	iq	2010	23	2010-06-11	0.247057	0.146057	0.250357	0.233714	58.94	296.598571	...	50.60	94.280000	58.94	17.008571	7.500000	24.400000	6.900000	32.2	19.2	7.4
1454	iq	2010	24	2010-06-18	0.333914	0.245771	0.278886	0.325486	59.67	296.345714	...	62.33	94.660000	59.67	16.815714	7.871429	25.433333	8.733333	31.2	21.0	16.0
1455	iq	2010	25	2010-06-25	0.298186	0.232971	0.274214	0.315757	63.22	298.097143	...	36.90	89.082857	63.22	17.355714	11.014286	27.475000	9.900000	33.7	22.2	20.4

	ndvi_ne	ndvi_nw	ndvi_se	ndvi_sw	precipitation_amt_mm	reanalysis_air_temp_k	reanalysis_avg_temp_k	reanalysis_dew_point_temp_k	reanalysis_max_air_temp_k	reanalysis_min_air_temp_k	reanalysis_precip_amt_kg_per_m2	reanalysis_relative_humidity_percent	reanalysis_sat_precip_amt_mm	reanalysis_specific_humidity_g_per_kg	reanalysis_tdtr_k	station_avg_temp_c	station_diur_temp_rng_c	station_max_temp_c	station_min_temp_c	station_precip_mm
count	1262.000000	1404.000000	1434.000000	1434.000000	1443.000000	1446.000000	1446.000000	1446.000000	1446.000000	1446.000000	1446.000000	1446.000000	1443.000000	1446.000000	1446.000000	1413.000000	1413.000000	1436.000000	1442.000000	1434.000000
mean	0.142294	0.130553	0.203783	0.202305	45.760388	298.701852	299.225578	295.246356	303.427109	295.719156	40.151819	82.161959	45.760388	16.746427	4.903754	27.185783	8.059328	32.452437	22.102150	39.326360
std	0.140531	0.119999	0.073860	0.083903	43.715537	1.362420	1.261715	1.527810	3.234601	2.565364	43.434399	7.153897	43.715537	1.542494	3.546445	1.292347	2.128568	1.959318	1.574066	47.455314
min	-0.406250	-0.456100	-0.015533	-0.063457	0.000000	294.635714	294.892857	289.642857	297.800000	286.900000	0.000000	57.787143	0.000000	11.715714	1.357143	21.400000	4.528571	26.700000	14.700000	0.000000
25%	0.044950	0.049217	0.155087	0.144209	9.800000	297.658929	298.257143	294.118929	301.000000	293.900000	13.055000	77.177143	9.800000	15.557143	2.328571	26.300000	6.514286	31.100000	21.100000	8.700000
50%	0.128817	0.121429	0.196050	0.189450	38.340000	298.646429	299.289286	295.640714	302.400000	296.200000	27.245000	80.301429	38.340000	17.087143	2.857143	27.414286	7.300000	32.800000	22.200000	23.850000
75%	0.248483	0.216600	0.248846	0.246982	70.235000	299.833571	300.207143	296.460000	305.500000	297.900000	52.200000	86.357857	70.235000	17.978214	7.625000	28.157143	9.566667	33.900000	23.300000	53.900000
max	0.508357	0.454429	0.538314	0.546017	390.600000	302.200000	302.928571	298.450000	314.000000	299.900000	570.500000	98.610000	390.600000	20.461429	16.028571	30.800000	15.800000	42.200000	25.600000	543.300000

	year	weekofyear	week_start_date	ndvi_ne	ndvi_nw	ndvi_se	ndvi_sw	precipitation_amt_mm	reanalysis_air_temp_k	...	reanalysis_precip_amt_kg_per_m2	reanalysis_relative_humidity_percent	reanalysis_sat_precip_amt_mm	reanalysis_specific_humidity_g_per_kg	reanalysis_tdtr_k	station_avg_temp_c	station_diur_temp_rng_c	station_max_temp_c	station_min_temp_c	station_precip_mm
0	1990	18	1990-04-30	0.122600	0.103725	0.198483	0.177617	12.42	297.572857	...	32.00	73.365714	12.42	14.012857	2.628571	25.442857	6.900000	29.4	20.0	16.0
1	1990	19	1990-05-07	0.169900	0.142175	0.162357	0.155486	22.82	298.211429	...	17.94	77.368571	22.82	15.372857	2.371429	26.714286	6.371429	31.7	22.2	8.6
2	1990	20	1990-05-14	0.032250	0.172967	0.157200	0.170843	34.54	298.781429	...	26.10	82.052857	34.54	16.848571	2.300000	26.714286	6.485714	32.2	22.8	41.4
3	1990	21	1990-05-21	0.128633	0.245067	0.227557	0.235886	15.36	298.987143	...	13.90	80.337143	15.36	16.672857	2.428571	27.471429	6.771429	33.3	23.3	4.0
4	1990	22	1990-05-28	0.196200	0.262200	0.251200	0.247340	7.52	299.518571	...	12.20	80.460000	7.52	17.210000	3.014286	28.942857	9.371429	35.0	23.9	5.8

	ndvi_ne	ndvi_nw	ndvi_se	ndvi_sw	reanalysis_air_temp_k	reanalysis_avg_temp_k	reanalysis_max_air_temp_k	reanalysis_min_air_temp_k	reanalysis_precip_amt_kg_per_m2	reanalysis_relative_humidity_percent	reanalysis_sat_precip_amt_mm	reanalysis_specific_humidity_g_per_kg	reanalysis_tdtr_k	station_avg_temp_c	station_diur_temp_rng_c	station_max_temp_c	station_min_temp_c	station_precip_mm
count	745.000000	887.000000	917.000000	917.000000	930.000000	930.000000	930.000000	930.000000	930.000000	930.000000	927.000000	930.000000	930.000000	930.000000	930.000000	930.000000	930.000000	930.000000
mean	0.057925	0.067469	0.177655	0.165956	299.163653	299.276920	301.398817	297.301828	30.465419	78.568181	35.470809	16.552409	2.516267	27.006528	6.757373	31.607957	22.600645	26.785484
std	0.107153	0.092479	0.057166	0.056073	1.236429	1.218637	1.258927	1.294705	35.628055	3.389488	44.606137	1.560923	0.498892	1.415473	0.835993	1.717297	1.506277	29.325811
min	-0.406250	-0.456100	-0.015533	-0.063457	295.938571	296.114286	297.800000	292.600000	0.000000	66.735714	0.000000	11.715714	1.357143	22.842857	4.528571	26.700000	17.800000	0.000000
25%	0.004500	0.016425	0.139283	0.129157	298.195000	298.300000	300.400000	296.300000	10.825000	76.246071	0.000000	15.236429	2.157143	25.842857	6.200000	30.600000	21.700000	6.825000
50%	0.057700	0.068075	0.177186	0.165971	299.254286	299.378571	301.500000	297.500000	21.300000	78.667857	20.800000	16.845714	2.457143	27.228571	6.757143	31.700000	22.800000	17.750000
75%	0.111100	0.115200	0.212557	0.202771	300.132857	300.228571	302.400000	298.400000	37.000000	80.963214	52.180000	17.858571	2.800000	28.185714	7.285714	32.800000	23.900000	35.450000
max	0.493400	0.437100	0.393129	0.381420	302.200000	302.164286	304.300000	299.900000	570.500000	87.575714	390.600000	19.440000	4.428571	30.071429	9.914286	35.600000	25.600000	305.900000

PRÁCTICA 1: APRENDIZAJE NO SUPERVISADO (DengAI)

Resumen¶

Contexto¶

Objetivo¶

Fases experimentación y resultados obtenidos¶

Inicialización¶

Exploración¶

Análisis Exploratorio de Datos (EDA)¶

Dataset completo¶

Estadísticas generales iniciales¶

Diagrama de barras variables categóricas¶

Transformación variable categórica a numérica¶

Análisis univariante (histogramas/percentiles)¶

Análisis bivariante (diagramas de dispersión 2 a 2)¶

Correlaciones de características¶

Gráficas de evolución temporal¶

Dataset dividido por ciudades (sj e iq)¶

Estadísticas generales San Juan e Iquitos¶

Percentiles y correlaciones San Juan e Iquitos¶

Preprocesado y transformación¶

Estudio y tratamiento de valores nulos¶

Estudio y tratamiento de outliers¶

Estadísticas generales post preprocesado¶

Características¶

Creación de características¶

Selección de características mediante clustering¶

Dataset traspuesto¶

Análisis de componentes (PCA)¶

Clustering jerárquico¶

DBSCAN¶

Clustering¶

Estandarización/Normalización de los datos¶

K-Means¶

Clustering jerárquico aglomerativo¶

DBSCAN¶

Expectation - Maximization aplicado a Gaussian Mixture Models (GMM)¶

Conclusiones¶

	ndvi_ne	ndvi_nw	ndvi_se	ndvi_sw	reanalysis_air_temp_k	reanalysis_avg_temp_k	reanalysis_max_air_temp_k	reanalysis_min_air_temp_k	reanalysis_precip_amt_kg_per_m2	reanalysis_relative_humidity_percent	reanalysis_sat_precip_amt_mm	reanalysis_specific_humidity_g_per_kg	reanalysis_tdtr_k	station_avg_temp_c	station_diur_temp_rng_c	station_max_temp_c	station_min_temp_c	station_precip_mm
count	517.000000	517.000000	517.000000	517.000000	516.000000	516.000000	516.000000	516.000000	516.000000	516.000000	516.000000	516.000000	516.000000	483.000000	483.000000	506.000000	512.000000	504.000000
mean	0.263869	0.238783	0.250126	0.266779	297.869538	299.133043	307.082752	292.866667	57.609864	88.639117	64.245736	17.096110	9.206783	27.530933	10.566197	34.004545	21.196680	62.467262
std	0.081370	0.076751	0.077354	0.086345	1.170997	1.332073	2.382980	1.663069	50.286555	7.583889	35.218995	1.445769	2.448525	0.921769	1.535496	1.325261	1.260327	63.245958
min	0.061729	0.035860	0.029880	0.064183	294.635714	294.892857	300.000000	286.900000	0.000000	57.787143	0.000000	12.111429	3.714286	21.400000	5.200000	30.100000	14.700000	0.000000
25%	0.200000	0.179540	0.194743	0.204129	297.092500	298.221429	305.200000	291.975000	24.065000	84.295000	39.105000	16.102857	7.371429	27.000000	9.500000	33.200000	20.600000	17.200000
50%	0.263643	0.232971	0.249800	0.262143	297.822857	299.121429	307.050000	293.050000	46.440000	90.917143	60.470000	17.428571	8.964286	27.600000	10.625000	34.000000	21.300000	45.300000
75%	0.319971	0.293929	0.302300	0.325150	298.649286	300.123214	308.700000	294.200000	71.072500	94.563929	85.757500	18.180357	11.014286	28.100000	11.655000	34.900000	22.000000	85.950000
max	0.508357	0.454429	0.538314	0.546017	301.637143	302.928571	314.000000	296.000000	362.030000	98.610000	210.830000	20.461429	16.028571	30.800000	15.800000	42.200000	24.200000	543.300000

	ndvi_ne	ndvi_nw	ndvi_se	ndvi_sw	reanalysis_air_temp_k	reanalysis_avg_temp_k	reanalysis_max_air_temp_k	reanalysis_min_air_temp_k	reanalysis_precip_amt_kg_per_m2	reanalysis_relative_humidity_percent	reanalysis_sat_precip_amt_mm	reanalysis_specific_humidity_g_per_kg	reanalysis_tdtr_k	station_avg_temp_c	station_diur_temp_rng_c	station_max_temp_c	station_min_temp_c	station_precip_mm
count	1451.000000	1451.000000	1451.000000	1451.000000	1451.000000	1451.000000	1451.000000	1451.000000	1451.000000	1451.000000	1451.000000	1451.000000	1451.000000	1451.000000	1451.000000	1451.000000	1451.000000	1451.000000
mean	0.131656	0.128011	0.203166	0.202092	298.699498	299.223156	303.422743	295.718677	40.151733	82.167472	45.581158	16.745225	4.901398	27.165660	1.881729	32.453549	22.096382	39.422467
std	0.137188	0.119338	0.073723	0.083468	1.361537	1.261339	3.233143	2.562014	43.381489	7.156206	43.694504	1.541798	3.543842	1.299260	40.565977	1.959432	1.572373	47.233605
min	-0.406250	-0.456100	-0.015533	-0.063457	294.635714	294.892857	297.800000	286.900000	0.000000	57.787143	0.000000	11.715714	1.357143	21.400000	-267.621429	26.700000	14.700000	0.000000
25%	0.037475	0.048100	0.153986	0.144600	297.657143	298.257143	301.000000	293.900000	13.150000	77.181429	9.630000	15.553571	2.328571	26.271429	6.442857	31.100000	21.100000	8.700000
50%	0.115700	0.115967	0.195729	0.190143	298.642857	299.285714	302.400000	296.200000	27.290000	80.300000	38.170000	17.085714	2.857143	27.400000	7.242857	32.800000	22.200000	24.100000
75%	0.231457	0.213086	0.247536	0.246629	299.830000	300.207143	305.500000	297.900000	52.350000	86.407143	70.115000	17.976429	7.621429	28.142857	9.500000	33.900000	23.300000	54.100000
max	0.508357	0.454429	0.538314	0.546017	302.200000	302.928571	314.000000	299.900000	570.500000	98.610000	390.600000	20.461429	16.028571	30.800000	15.800000	42.200000	25.600000	543.300000

	0	1	2	3	4	5	6	7	8	9	...	1446	1447	1448	1449	1450	1451	1452	1453	1454	1455
city	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	...	1.000000	1.000000	1.000000	1.000000	1.000000	1.000000	1.000000	1.000000	1.000000	1.000000
weekofyear	18.000000	19.000000	20.000000	21.000000	22.000000	23.000000	24.000000	25.000000	26.000000	27.000000	...	16.000000	17.000000	18.000000	19.000000	20.000000	21.000000	22.000000	23.000000	24.000000	25.000000
ndvi_ne	0.122600	0.169900	0.032250	0.128633	0.196200	0.154550	0.112900	0.072500	0.102450	0.147663	...	0.231486	0.239743	0.260814	0.168686	0.263071	0.342750	0.160157	0.247057	0.333914	0.298186
ndvi_nw	0.103725	0.142175	0.172967	0.245067	0.262200	0.174850	0.092800	0.072500	0.146175	0.121550	...	0.294686	0.259271	0.255786	0.158500	0.272500	0.318900	0.160371	0.146057	0.245771	0.232971
ndvi_se	0.198483	0.162357	0.157200	0.227557	0.251200	0.254314	0.205071	0.151471	0.125571	0.160683	...	0.331657	0.307786	0.257771	0.133071	0.258271	0.256343	0.136043	0.250357	0.278886	0.274214
ndvi_sw	0.177617	0.155486	0.170843	0.235886	0.247340	0.181743	0.210271	0.133029	0.123600	0.202567	...	0.244400	0.307943	0.340286	0.145600	0.244500	0.292514	0.225657	0.233714	0.325486	0.315757
reanalysis_air_temp_k	297.572857	298.211429	298.781429	298.987143	299.518571	299.630000	299.207143	299.591429	299.578571	300.154286	...	298.438571	299.048571	297.617143	297.278571	297.648571	299.334286	298.330000	296.598571	296.345714	298.097143
reanalysis_avg_temp_k	297.742857	298.442857	298.878571	299.228571	299.664286	299.764286	299.221429	299.528571	299.557143	300.278571	...	299.507143	300.028571	298.585714	297.935714	298.707143	300.771429	299.392857	297.592857	297.521429	299.835714
reanalysis_max_air_temp_k	299.800000	300.900000	300.500000	301.400000	301.900000	302.400000	301.300000	300.600000	302.100000	302.300000	...	304.700000	308.400000	304.700000	306.000000	308.700000	309.700000	308.500000	305.500000	306.100000	307.800000
reanalysis_min_air_temp_k	295.900000	296.400000	297.300000	297.000000	297.500000	298.100000	297.700000	298.400000	297.700000	298.700000	...	294.700000	294.600000	294.600000	294.000000	290.100000	294.500000	291.900000	292.400000	291.900000	292.300000
reanalysis_precip_amt_kg_per_m2	32.000000	17.940000	26.100000	13.900000	12.200000	26.490000	38.600000	30.000000	37.510000	28.400000	...	81.400000	23.600000	85.460000	87.300000	8.800000	45.000000	207.100000	50.600000	62.330000	36.900000
reanalysis_relative_humidity_percent	73.365714	77.368571	82.052857	80.337143	80.460000	79.891429	82.000000	83.375714	82.768571	81.281429	...	95.995714	87.657143	96.712857	97.445714	78.998571	88.765714	91.600000	94.280000	94.660000	89.082857
reanalysis_sat_precip_amt_mm	12.420000	22.820000	34.540000	15.360000	7.520000	9.580000	3.480000	151.120000	19.320000	14.410000	...	86.700000	26.000000	73.970000	59.400000	1.150000	55.300000	86.470000	58.940000	59.670000	63.220000
reanalysis_specific_humidity_g_per_kg	14.012857	15.372857	16.848571	16.672857	17.210000	17.212857	17.234286	17.977143	17.790000	18.071429	...	19.448571	18.068571	18.602857	18.391429	14.908571	18.485714	18.070000	17.008571	16.815714	17.355714
reanalysis_tdtr_k	2.628571	2.371429	2.300000	2.428571	3.014286	2.100000	2.042857	1.571429	1.885714	2.014286	...	7.757143	8.257143	5.714286	6.185714	11.242857	9.800000	7.471429	7.500000	7.871429	11.014286
station_avg_temp_c	25.442857	26.714286	26.714286	27.471429	28.942857	28.114286	27.414286	28.371429	28.328571	28.328571	...	27.850000	28.850000	27.600000	27.400000	25.633333	28.633333	27.433333	24.400000	25.433333	27.475000
station_diur_temp_rng_c	6.900000	6.371429	6.485714	6.771429	9.371429	6.942857	6.771429	7.685714	7.385714	6.514286	...	9.600000	12.125000	9.600000	10.400000	9.200000	11.933333	10.500000	6.900000	8.733333	9.900000
station_max_temp_c	29.400000	31.700000	32.200000	33.300000	35.000000	34.400000	32.200000	33.900000	33.900000	33.900000	...	33.500000	36.200000	33.200000	33.700000	34.000000	35.400000	34.700000	32.200000	31.200000	33.700000
station_min_temp_c	20.000000	22.200000	22.800000	23.300000	23.900000	23.900000	23.300000	22.800000	22.800000	24.400000	...	22.500000	21.400000	21.400000	21.200000	20.000000	22.400000	21.700000	19.200000	21.000000	22.200000
station_precip_mm	16.000000	8.600000	41.400000	4.000000	5.800000	39.100000	29.700000	21.100000	21.100000	1.100000	...	51.100000	35.400000	8.100000	32.000000	2.500000	27.000000	36.600000	7.400000	16.000000	20.400000
avg_ndvi	0.150606	0.157479	0.133315	0.209286	0.239235	0.191364	0.155261	0.107375	0.124449	0.158116	...	0.275557	0.278686	0.278664	0.151464	0.259586	0.302627	0.170557	0.219296	0.296014	0.280282
rng_ndvi	0.094758	0.027725	0.140717	0.116433	0.066000	0.099764	0.117471	0.078971	0.043725	0.081017	...	0.100171	0.068200	0.084500	0.035614	0.028000	0.086407	0.089614	0.104300	0.088143	0.082786
precip_avg	24.000000	13.270000	33.750000	8.950000	9.000000	32.795000	34.150000	25.550000	29.305000	14.750000	...	66.250000	29.500000	46.780000	59.650000	5.650000	36.000000	121.850000	29.000000	39.165000	28.650000